#!/usr/bin/python2.4
#
# Copyright 2011 Google Inc. All Rights Reserved.

"""This module retrieves tweets and adds them to the datastore."""

__author__ = 'ryanboyd@google.com'

from google.appengine.ext import webapp
from google.appengine.ext.webapp.util import run_wsgi_app
from google.appengine.ext import db
from django.utils import simplejson
from google.appengine.api import urlfetch
import logging
import model

class MainPage(webapp.RequestHandler):
  def get(self):
    self.response.headers['Content-Type'] = 'text/plain'
    self.response.out.write('Adding tweets: ')
    tf = TwitterFetcher()
    self.response.out.write(tf.conditionalAddTweetSet())

application = webapp.WSGIApplication(
    [('/tasks/twitter_fetch', MainPage)],
    debug=True)

class TwitterFetcher():
  def fetchContent(self, url):        
    try:            
      fetchResult = urlfetch.fetch(url)        
    except Error:            
      logging.error('Exception: Error retrieving tweets')
      return
    if fetchResult.status_code == 200:            
      encTypeRaw = fetchResult.headers['content-type']            
      encStartIdx = encTypeRaw.find('=')            
      encType = encTypeRaw[encStartIdx + 1:]            
      fetchResultContent = fetchResult.content            
      return fetchResultContent.decode(encType)        
    return 'Warning: URL could not be retrieved successfully'
  
  def retrieveTweetSet(self):
    url = "http://search.twitter.com/search.json?q=the"
    fetchResult = self.fetchContent(url)
    jsonResult = simplejson.loads(fetchResult)    
    return jsonResult

  def conditionalAddTweetSet(self):
    tweets = db.GqlQuery("SELECT * FROM Item WHERE classified=NULL")
    if tweets.count() < 1000:
      return self.addTweetSet()
    else:
      return -1

  def addTweetSet(self):
    rs = self.retrieveTweetSet()
    i = 0
    for tweet in rs['results']:
      urlstr = 'http://twitter.com/%s/status/%s' % (tweet['from_user'], tweet['id'])
      query = model.Item.all()
      query.filter('url = ', urlstr)
      data = query.fetch(1)
      if not data:
        item = model.Item()
        item.url = urlstr
        item.data = tweet['text']
        item.classified = False
        item.put() 
        i += 1
    return i
  
def main():
  logging.getLogger().setLevel(logging.DEBUG)
  run_wsgi_app(application)

if __name__ == "__main__":
  main()
